{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from scipy import stats\n",
    "import warnings;\n",
    "from pysqldf import SQLDF\n",
    "import pandasql as psql\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "from sklearn.model_selection import KFold\n",
    "import sklearn.ensemble as ske\n",
    "import lightgbm as lgb\n",
    "from pandas.api.types import is_string_dtype\n",
    "from pandas.api.types import is_numeric_dtype\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TrnDataForLGB.csv\")\n",
    "test01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TstDataForLGB.csv\")\n",
    "\n",
    "Remove_List = [\"id\",\"Price\",\"Name\",\"Lag_Price2_MIN\",\"Lag_Price2_MAX\",\n",
    "               \"Lag_Price3_MIN\",\"Lag_Price3_MAX\",\"Lag_Price\",\"Lag_Price3\",\"Engine_Group\",\n",
    "               \"Power_Group\",\"TrainTestInd\",\"CarCompName\",\"RateChng1\",\"RateChng2\",\"RateChng3\",\n",
    "               \"Lag_Price4_MIN\",\"Lag_Price4_MAX\",\"Lag_Price4_MIN_BY_MAX\"]\n",
    "feature_names = list(set(list(train01.columns)) - set(Remove_List))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Lag_Price4', 'Lag_Price2', 'New_Price'], dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train01[feature_names].columns[train01[feature_names].isnull().any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Lag_Price4', 'Lag_Price2', 'New_Price'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test01[feature_names].columns[test01[feature_names].isnull().any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train01['Lag_Price4'] = train01['Lag_Price4'].fillna(value=0)\n",
    "train01['Lag_Price2'] = train01['Lag_Price2'].fillna(value=0)\n",
    "train01['New_Price'] = train01['New_Price'].fillna(value=0)\n",
    "\n",
    "test01['Lag_Price4'] = test01['Lag_Price4'].fillna(value=0)\n",
    "test01['Lag_Price2'] = test01['Lag_Price2'].fillna(value=0)\n",
    "test01['New_Price'] = test01['New_Price'].fillna(value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([], dtype='object')\n",
      "Index([], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(train01[feature_names].columns[train01[feature_names].isnull().any()])\n",
    "print(test01[feature_names].columns[test01[feature_names].isnull().any()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train02 = train01[feature_names]\n",
    "y_train02 = train01['Price']\n",
    "x_sub2 = test01[feature_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fuel_TypeElectric</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.000332</td>\n",
       "      <td>0.018227</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_MOBILIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002658</td>\n",
       "      <td>0.051494</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kilometers_Driven</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>58738.380296</td>\n",
       "      <td>91268.843206</td>\n",
       "      <td>171.0</td>\n",
       "      <td>34000.00</td>\n",
       "      <td>53000.00</td>\n",
       "      <td>73000.0</td>\n",
       "      <td>6500000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_VENTO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.017777</td>\n",
       "      <td>0.132151</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ECOSPORT</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.015783</td>\n",
       "      <td>0.124647</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_LAND</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010301</td>\n",
       "      <td>0.100977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_DZIRE</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.029241</td>\n",
       "      <td>0.168495</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ACCENT</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002991</td>\n",
       "      <td>0.054608</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_BRIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010135</td>\n",
       "      <td>0.100168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fuel_TypePetrol</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.456222</td>\n",
       "      <td>0.498121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_VOLVO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.003489</td>\n",
       "      <td>0.058969</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Seats</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>5.277122</td>\n",
       "      <td>0.806644</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_JEEP</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002824</td>\n",
       "      <td>0.053074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_VITARA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.007809</td>\n",
       "      <td>0.088028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Owner_TypeFirst</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.818907</td>\n",
       "      <td>0.385127</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_A6</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.008141</td>\n",
       "      <td>0.089866</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationMumbai</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.131251</td>\n",
       "      <td>0.337703</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CELERIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.009636</td>\n",
       "      <td>0.097698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_HONDA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.101013</td>\n",
       "      <td>0.301372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_MINI</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.065587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_XUV500</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.018774</td>\n",
       "      <td>0.135737</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_SKODA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.028742</td>\n",
       "      <td>0.167095</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CIVIC</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.005316</td>\n",
       "      <td>0.072726</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CR-V</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004486</td>\n",
       "      <td>0.066831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_MANZA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002658</td>\n",
       "      <td>0.051494</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_5</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.168134</td>\n",
       "      <td>0.374017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CIAZ</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.011796</td>\n",
       "      <td>0.107976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_FIGO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.016780</td>\n",
       "      <td>0.128458</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_INNOVA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.027247</td>\n",
       "      <td>0.162816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_WAGON</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.025586</td>\n",
       "      <td>0.157909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_LINEA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001994</td>\n",
       "      <td>0.044610</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_IKON</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002492</td>\n",
       "      <td>0.049863</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_MICRA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.005483</td>\n",
       "      <td>0.073848</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_EON</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010135</td>\n",
       "      <td>0.100168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_GRAND</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.027081</td>\n",
       "      <td>0.162333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_LAURA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004818</td>\n",
       "      <td>0.069251</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_I10</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.051171</td>\n",
       "      <td>0.220365</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mileage</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>17.796986</td>\n",
       "      <td>19.097099</td>\n",
       "      <td>-999.0</td>\n",
       "      <td>15.16</td>\n",
       "      <td>18.15</td>\n",
       "      <td>21.1</td>\n",
       "      <td>33.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_GL-CLASS</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001828</td>\n",
       "      <td>0.042714</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_X5</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004486</td>\n",
       "      <td>0.066831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Power</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>112.668894</td>\n",
       "      <td>53.940547</td>\n",
       "      <td>34.2</td>\n",
       "      <td>74.00</td>\n",
       "      <td>93.70</td>\n",
       "      <td>138.1</td>\n",
       "      <td>560.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SANTRO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.012959</td>\n",
       "      <td>0.113107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Owner_TypeFourth...Above</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001495</td>\n",
       "      <td>0.038643</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_CHEVROLET</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.020103</td>\n",
       "      <td>0.140364</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_S</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.689649</td>\n",
       "      <td>0.462676</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SSANGYONG</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002160</td>\n",
       "      <td>0.046428</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_A4</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.012959</td>\n",
       "      <td>0.113107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_VOLKSWAGEN</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.052334</td>\n",
       "      <td>0.222719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationHyderabad</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.123276</td>\n",
       "      <td>0.328781</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_MAHINDRA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.045190</td>\n",
       "      <td>0.207738</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationBangalore</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.059478</td>\n",
       "      <td>0.236537</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_MITSUBISHI</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004486</td>\n",
       "      <td>0.066831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SUNNY</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.065587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_BALENO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010467</td>\n",
       "      <td>0.101779</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_COMPASS</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002492</td>\n",
       "      <td>0.049863</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_FORTUNER</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.016780</td>\n",
       "      <td>0.128458</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_Q5</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004984</td>\n",
       "      <td>0.070429</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TransmissionManual</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.714238</td>\n",
       "      <td>0.451814</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fuel_TypeDiesel</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.532480</td>\n",
       "      <td>0.498985</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CRETA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.015451</td>\n",
       "      <td>0.123349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>157 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            count          mean           std    min  \\\n",
       "Fuel_TypeElectric          6019.0      0.000332      0.018227    0.0   \n",
       "CompNameCarName_MOBILIO    6019.0      0.002658      0.051494    0.0   \n",
       "Kilometers_Driven          6019.0  58738.380296  91268.843206  171.0   \n",
       "CompNameCarName_VENTO      6019.0      0.017777      0.132151    0.0   \n",
       "CompNameCarName_ECOSPORT   6019.0      0.015783      0.124647    0.0   \n",
       "CompName_LAND              6019.0      0.010301      0.100977    0.0   \n",
       "CompNameCarName_DZIRE      6019.0      0.029241      0.168495    0.0   \n",
       "CompNameCarName_ACCENT     6019.0      0.002991      0.054608    0.0   \n",
       "CompNameCarName_BRIO       6019.0      0.010135      0.100168    0.0   \n",
       "Fuel_TypePetrol            6019.0      0.456222      0.498121    0.0   \n",
       "CompName_VOLVO             6019.0      0.003489      0.058969    0.0   \n",
       "Seats                      6019.0      5.277122      0.806644    0.0   \n",
       "CompName_JEEP              6019.0      0.002824      0.053074    0.0   \n",
       "CompNameCarName_VITARA     6019.0      0.007809      0.088028    0.0   \n",
       "Owner_TypeFirst            6019.0      0.818907      0.385127    0.0   \n",
       "CompNameCarName_A6         6019.0      0.008141      0.089866    0.0   \n",
       "LocationMumbai             6019.0      0.131251      0.337703    0.0   \n",
       "CompNameCarName_CELERIO    6019.0      0.009636      0.097698    0.0   \n",
       "CompName_HONDA             6019.0      0.101013      0.301372    0.0   \n",
       "CompName_MINI              6019.0      0.004320      0.065587    0.0   \n",
       "CompNameCarName_XUV500     6019.0      0.018774      0.135737    0.0   \n",
       "CompName_SKODA             6019.0      0.028742      0.167095    0.0   \n",
       "CompNameCarName_CIVIC      6019.0      0.005316      0.072726    0.0   \n",
       "CompNameCarName_CR-V       6019.0      0.004486      0.066831    0.0   \n",
       "CompNameCarName_MANZA      6019.0      0.002658      0.051494    0.0   \n",
       "CompNameCarName_5          6019.0      0.168134      0.374017    0.0   \n",
       "CompNameCarName_CIAZ       6019.0      0.011796      0.107976    0.0   \n",
       "CompNameCarName_FIGO       6019.0      0.016780      0.128458    0.0   \n",
       "CompNameCarName_INNOVA     6019.0      0.027247      0.162816    0.0   \n",
       "CompNameCarName_WAGON      6019.0      0.025586      0.157909    0.0   \n",
       "...                           ...           ...           ...    ...   \n",
       "CompNameCarName_LINEA      6019.0      0.001994      0.044610    0.0   \n",
       "CompNameCarName_IKON       6019.0      0.002492      0.049863    0.0   \n",
       "CompNameCarName_MICRA      6019.0      0.005483      0.073848    0.0   \n",
       "CompNameCarName_EON        6019.0      0.010135      0.100168    0.0   \n",
       "CompNameCarName_GRAND      6019.0      0.027081      0.162333    0.0   \n",
       "CompNameCarName_LAURA      6019.0      0.004818      0.069251    0.0   \n",
       "CompNameCarName_I10        6019.0      0.051171      0.220365    0.0   \n",
       "Mileage                    6019.0     17.796986     19.097099 -999.0   \n",
       "CompNameCarName_GL-CLASS   6019.0      0.001828      0.042714    0.0   \n",
       "CompNameCarName_X5         6019.0      0.004486      0.066831    0.0   \n",
       "Power                      6019.0    112.668894     53.940547   34.2   \n",
       "CompNameCarName_SANTRO     6019.0      0.012959      0.113107    0.0   \n",
       "Owner_TypeFourth...Above   6019.0      0.001495      0.038643    0.0   \n",
       "CompName_CHEVROLET         6019.0      0.020103      0.140364    0.0   \n",
       "CompNameCarName_S          6019.0      0.689649      0.462676    0.0   \n",
       "CompNameCarName_SSANGYONG  6019.0      0.002160      0.046428    0.0   \n",
       "CompNameCarName_A4         6019.0      0.012959      0.113107    0.0   \n",
       "CompName_VOLKSWAGEN        6019.0      0.052334      0.222719    0.0   \n",
       "LocationHyderabad          6019.0      0.123276      0.328781    0.0   \n",
       "CompName_MAHINDRA          6019.0      0.045190      0.207738    0.0   \n",
       "LocationBangalore          6019.0      0.059478      0.236537    0.0   \n",
       "CompName_MITSUBISHI        6019.0      0.004486      0.066831    0.0   \n",
       "CompNameCarName_SUNNY      6019.0      0.004320      0.065587    0.0   \n",
       "CompNameCarName_BALENO     6019.0      0.010467      0.101779    0.0   \n",
       "CompNameCarName_COMPASS    6019.0      0.002492      0.049863    0.0   \n",
       "CompNameCarName_FORTUNER   6019.0      0.016780      0.128458    0.0   \n",
       "CompNameCarName_Q5         6019.0      0.004984      0.070429    0.0   \n",
       "TransmissionManual         6019.0      0.714238      0.451814    0.0   \n",
       "Fuel_TypeDiesel            6019.0      0.532480      0.498985    0.0   \n",
       "CompNameCarName_CRETA      6019.0      0.015451      0.123349    0.0   \n",
       "\n",
       "                                25%       50%      75%         max  \n",
       "Fuel_TypeElectric              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_MOBILIO        0.00      0.00      0.0        1.00  \n",
       "Kilometers_Driven          34000.00  53000.00  73000.0  6500000.00  \n",
       "CompNameCarName_VENTO          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ECOSPORT       0.00      0.00      0.0        1.00  \n",
       "CompName_LAND                  0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_DZIRE          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ACCENT         0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_BRIO           0.00      0.00      0.0        1.00  \n",
       "Fuel_TypePetrol                0.00      0.00      1.0        1.00  \n",
       "CompName_VOLVO                 0.00      0.00      0.0        1.00  \n",
       "Seats                          5.00      5.00      5.0       10.00  \n",
       "CompName_JEEP                  0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_VITARA         0.00      0.00      0.0        1.00  \n",
       "Owner_TypeFirst                1.00      1.00      1.0        1.00  \n",
       "CompNameCarName_A6             0.00      0.00      0.0        1.00  \n",
       "LocationMumbai                 0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CELERIO        0.00      0.00      0.0        1.00  \n",
       "CompName_HONDA                 0.00      0.00      0.0        1.00  \n",
       "CompName_MINI                  0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_XUV500         0.00      0.00      0.0        1.00  \n",
       "CompName_SKODA                 0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CIVIC          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CR-V           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_MANZA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_5              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CIAZ           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_FIGO           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_INNOVA         0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_WAGON          0.00      0.00      0.0        1.00  \n",
       "...                             ...       ...      ...         ...  \n",
       "CompNameCarName_LINEA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_IKON           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_MICRA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_EON            0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_GRAND          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_LAURA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_I10            0.00      0.00      0.0        1.00  \n",
       "Mileage                       15.16     18.15     21.1       33.54  \n",
       "CompNameCarName_GL-CLASS       0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_X5             0.00      0.00      0.0        1.00  \n",
       "Power                         74.00     93.70    138.1      560.00  \n",
       "CompNameCarName_SANTRO         0.00      0.00      0.0        1.00  \n",
       "Owner_TypeFourth...Above       0.00      0.00      0.0        1.00  \n",
       "CompName_CHEVROLET             0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_S              0.00      1.00      1.0        1.00  \n",
       "CompNameCarName_SSANGYONG      0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_A4             0.00      0.00      0.0        1.00  \n",
       "CompName_VOLKSWAGEN            0.00      0.00      0.0        1.00  \n",
       "LocationHyderabad              0.00      0.00      0.0        1.00  \n",
       "CompName_MAHINDRA              0.00      0.00      0.0        1.00  \n",
       "LocationBangalore              0.00      0.00      0.0        1.00  \n",
       "CompName_MITSUBISHI            0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_SUNNY          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_BALENO         0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_COMPASS        0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_FORTUNER       0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_Q5             0.00      0.00      0.0        1.00  \n",
       "TransmissionManual             0.00      1.00      1.0        1.00  \n",
       "Fuel_TypeDiesel                0.00      1.00      1.0        1.00  \n",
       "CompNameCarName_CRETA          0.00      0.00      0.0        1.00  \n",
       "\n",
       "[157 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stats = x_train02.describe()\n",
    "train_stats = train_stats.transpose()\n",
    "train_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def norm(x):\n",
    "  return(x - train_stats['mean']) / train_stats['std']\n",
    "normed_x_train02 = norm(x_train02)\n",
    "normed_x_sub2 = norm(x_sub2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normed_x_train02.reset_index(drop = True, inplace = True)\n",
    "kf = KFold(n_splits = 5, shuffle = True, random_state = 100)\n",
    "kf.get_n_splits(normed_x_train02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers.advanced_activations import PReLU\n",
    "from keras import optimizers\n",
    "from keras.callbacks import EarlyStopping\n",
    "import math\n",
    "\n",
    "def nn_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(100, input_dim = normed_x_train02.shape[1], init = 'he_normal'))\n",
    "    model.add(Activation('tanh'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(100, init = 'he_normal'))\n",
    "    model.add(Activation('tanh'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(30, init = 'he_normal'))\n",
    "    model.add(Activation('tanh'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(1, init = 'he_normal'))\n",
    "    model.compile(loss = 'mean_squared_error', optimizer = 'Adam')\n",
    "    return(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running CV Iteration Num : 1\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/100\n",
      "4815/4815 [==============================] - 3s 520us/step - loss: 1.2029 - val_loss: 0.2436\n",
      "Epoch 2/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.3484 - val_loss: 0.1364\n",
      "Epoch 3/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.2443 - val_loss: 0.1142\n",
      "Epoch 4/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.1835 - val_loss: 0.1186\n",
      "Epoch 5/100\n",
      "4815/4815 [==============================] - 1s 305us/step - loss: 0.1519 - val_loss: 0.0863\n",
      "Epoch 6/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.1375 - val_loss: 0.0694\n",
      "Epoch 7/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.1273 - val_loss: 0.0663\n",
      "Epoch 8/100\n",
      "4815/4815 [==============================] - 1s 263us/step - loss: 0.1193 - val_loss: 0.0900\n",
      "Epoch 9/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.1072 - val_loss: 0.0685\n",
      "Epoch 10/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.1036 - val_loss: 0.0531\n",
      "Epoch 11/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0985 - val_loss: 0.0614\n",
      "Epoch 12/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0914 - val_loss: 0.0476\n",
      "Epoch 13/100\n",
      "4815/4815 [==============================] - 1s 304us/step - loss: 0.0847 - val_loss: 0.0517\n",
      "Epoch 14/100\n",
      "4815/4815 [==============================] - 1s 303us/step - loss: 0.0796 - val_loss: 0.0508\n",
      "Epoch 15/100\n",
      "4815/4815 [==============================] - 1s 290us/step - loss: 0.0747 - val_loss: 0.0440\n",
      "Epoch 16/100\n",
      "4815/4815 [==============================] - 1s 290us/step - loss: 0.0727 - val_loss: 0.0490\n",
      "Epoch 17/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0673 - val_loss: 0.0424\n",
      "Epoch 18/100\n",
      "4815/4815 [==============================] - 1s 294us/step - loss: 0.0657 - val_loss: 0.0393\n",
      "Epoch 19/100\n",
      "4815/4815 [==============================] - 1s 291us/step - loss: 0.0633 - val_loss: 0.0426\n",
      "Epoch 20/100\n",
      "4815/4815 [==============================] - 1s 294us/step - loss: 0.0614 - val_loss: 0.0504\n",
      "Epoch 21/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0592 - val_loss: 0.0339\n",
      "Epoch 22/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0555 - val_loss: 0.0369\n",
      "Epoch 23/100\n",
      "4815/4815 [==============================] - 1s 266us/step - loss: 0.0544 - val_loss: 0.0338\n",
      "Epoch 24/100\n",
      "4815/4815 [==============================] - 1s 277us/step - loss: 0.0525 - val_loss: 0.0413\n",
      "Epoch 25/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0509 - val_loss: 0.0367\n",
      "Epoch 26/100\n",
      "4815/4815 [==============================] - 1s 296us/step - loss: 0.0494 - val_loss: 0.0405\n",
      "Epoch 27/100\n",
      "4815/4815 [==============================] - 2s 312us/step - loss: 0.0494 - val_loss: 0.0333\n",
      "Epoch 28/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0478 - val_loss: 0.0380\n",
      "Epoch 29/100\n",
      "4815/4815 [==============================] - 1s 298us/step - loss: 0.0471 - val_loss: 0.0310\n",
      "Epoch 30/100\n",
      "4815/4815 [==============================] - 1s 288us/step - loss: 0.0468 - val_loss: 0.0309\n",
      "Epoch 31/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0467 - val_loss: 0.0297\n",
      "Epoch 32/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0452 - val_loss: 0.0334\n",
      "Epoch 33/100\n",
      "4815/4815 [==============================] - 1s 295us/step - loss: 0.0474 - val_loss: 0.0307\n",
      "Epoch 34/100\n",
      "4815/4815 [==============================] - 2s 313us/step - loss: 0.0442 - val_loss: 0.0312\n",
      "Epoch 35/100\n",
      "4815/4815 [==============================] - 1s 267us/step - loss: 0.0449 - val_loss: 0.0282\n",
      "Epoch 36/100\n",
      "4815/4815 [==============================] - 1s 282us/step - loss: 0.0441 - val_loss: 0.0287\n",
      "Epoch 37/100\n",
      "4815/4815 [==============================] - 1s 267us/step - loss: 0.0433 - val_loss: 0.0276\n",
      "Epoch 38/100\n",
      "4815/4815 [==============================] - 1s 265us/step - loss: 0.0436 - val_loss: 0.0277\n",
      "Epoch 39/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0427 - val_loss: 0.0256\n",
      "Epoch 40/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0439 - val_loss: 0.0305\n",
      "Epoch 41/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0433 - val_loss: 0.0284\n",
      "Epoch 42/100\n",
      "4815/4815 [==============================] - 1s 282us/step - loss: 0.0424 - val_loss: 0.0288\n",
      "Epoch 43/100\n",
      "4815/4815 [==============================] - 1s 305us/step - loss: 0.0422 - val_loss: 0.0288\n",
      "Epoch 44/100\n",
      "4815/4815 [==============================] - 2s 316us/step - loss: 0.0419 - val_loss: 0.0311\n",
      "Epoch 45/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0419 - val_loss: 0.0306\n",
      "Epoch 46/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0411 - val_loss: 0.0266\n",
      "Epoch 47/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0405 - val_loss: 0.0273\n",
      "Epoch 48/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0411 - val_loss: 0.0276\n",
      "Epoch 49/100\n",
      "4815/4815 [==============================] - 1s 293us/step - loss: 0.0411 - val_loss: 0.0264\n",
      "Epoch 00049: early stopping\n",
      "Test RMSE :  0.16263320942953444\n",
      "Running CV Iteration Num : 2\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/100\n",
      "4815/4815 [==============================] - 3s 535us/step - loss: 1.1449 - val_loss: 0.1898\n",
      "Epoch 2/100\n",
      "4815/4815 [==============================] - 1s 302us/step - loss: 0.3091 - val_loss: 0.1200\n",
      "Epoch 3/100\n",
      "4815/4815 [==============================] - 1s 303us/step - loss: 0.2184 - val_loss: 0.1117\n",
      "Epoch 4/100\n",
      "4815/4815 [==============================] - 1s 300us/step - loss: 0.1707 - val_loss: 0.0845\n",
      "Epoch 5/100\n",
      "4815/4815 [==============================] - 1s 289us/step - loss: 0.1527 - val_loss: 0.0927\n",
      "Epoch 6/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.1349 - val_loss: 0.0713\n",
      "Epoch 7/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.1246 - val_loss: 0.0709\n",
      "Epoch 8/100\n",
      "4815/4815 [==============================] - 1s 284us/step - loss: 0.1164 - val_loss: 0.0715\n",
      "Epoch 9/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.1103 - val_loss: 0.0540\n",
      "Epoch 10/100\n",
      "4815/4815 [==============================] - 1s 285us/step - loss: 0.1035 - val_loss: 0.0597\n",
      "Epoch 11/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0926 - val_loss: 0.0522\n",
      "Epoch 12/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0905 - val_loss: 0.0460\n",
      "Epoch 13/100\n",
      "4815/4815 [==============================] - 1s 287us/step - loss: 0.0867 - val_loss: 0.0475\n",
      "Epoch 14/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0781 - val_loss: 0.0435\n",
      "Epoch 15/100\n",
      "4815/4815 [==============================] - 1s 286us/step - loss: 0.0785 - val_loss: 0.0430\n",
      "Epoch 16/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0721 - val_loss: 0.0467\n",
      "Epoch 17/100\n",
      "4815/4815 [==============================] - 1s 284us/step - loss: 0.0719 - val_loss: 0.0444\n",
      "Epoch 18/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0657 - val_loss: 0.0438\n",
      "Epoch 19/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0631 - val_loss: 0.0447\n",
      "Epoch 20/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0600 - val_loss: 0.0413\n",
      "Epoch 21/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0588 - val_loss: 0.0415\n",
      "Epoch 22/100\n",
      "4815/4815 [==============================] - 1s 284us/step - loss: 0.0565 - val_loss: 0.0416\n",
      "Epoch 23/100\n",
      "4815/4815 [==============================] - 2s 327us/step - loss: 0.0546 - val_loss: 0.0386\n",
      "Epoch 24/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0522 - val_loss: 0.0357\n",
      "Epoch 25/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0497 - val_loss: 0.0368\n",
      "Epoch 26/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0498 - val_loss: 0.0382\n",
      "Epoch 27/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0490 - val_loss: 0.0369\n",
      "Epoch 28/100\n",
      "4815/4815 [==============================] - 1s 254us/step - loss: 0.0465 - val_loss: 0.0369\n",
      "Epoch 29/100\n",
      "4815/4815 [==============================] - 1s 259us/step - loss: 0.0495 - val_loss: 0.0361\n",
      "Epoch 30/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0470 - val_loss: 0.0361\n",
      "Epoch 31/100\n",
      "4815/4815 [==============================] - 1s 265us/step - loss: 0.0469 - val_loss: 0.0355\n",
      "Epoch 32/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0449 - val_loss: 0.0344\n",
      "Epoch 33/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0447 - val_loss: 0.0378\n",
      "Epoch 34/100\n",
      "4815/4815 [==============================] - 1s 268us/step - loss: 0.0433 - val_loss: 0.0345\n",
      "Epoch 35/100\n",
      "4815/4815 [==============================] - 1s 260us/step - loss: 0.0437 - val_loss: 0.0349\n",
      "Epoch 36/100\n",
      "4815/4815 [==============================] - 1s 259us/step - loss: 0.0433 - val_loss: 0.0347\n",
      "Epoch 37/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0457 - val_loss: 0.0350\n",
      "Epoch 38/100\n",
      "4815/4815 [==============================] - 1s 257us/step - loss: 0.0432 - val_loss: 0.0334\n",
      "Epoch 39/100\n",
      "4815/4815 [==============================] - 1s 252us/step - loss: 0.0428 - val_loss: 0.0314\n",
      "Epoch 40/100\n",
      "4815/4815 [==============================] - 1s 268us/step - loss: 0.0431 - val_loss: 0.0324\n",
      "Epoch 41/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0413 - val_loss: 0.0317\n",
      "Epoch 42/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0409 - val_loss: 0.0319\n",
      "Epoch 43/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0413 - val_loss: 0.0328\n",
      "Epoch 44/100\n",
      "4815/4815 [==============================] - 1s 263us/step - loss: 0.0420 - val_loss: 0.0340\n",
      "Epoch 45/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0403 - val_loss: 0.0314\n",
      "Epoch 46/100\n",
      "4815/4815 [==============================] - 1s 257us/step - loss: 0.0399 - val_loss: 0.0324\n",
      "Epoch 47/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0410 - val_loss: 0.0324\n",
      "Epoch 48/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.0424 - val_loss: 0.0327\n",
      "Epoch 49/100\n",
      "4815/4815 [==============================] - 1s 254us/step - loss: 0.0400 - val_loss: 0.0332\n",
      "Epoch 50/100\n",
      "4815/4815 [==============================] - 1s 268us/step - loss: 0.0407 - val_loss: 0.0341\n",
      "Epoch 51/100\n",
      "4815/4815 [==============================] - 1s 250us/step - loss: 0.0396 - val_loss: 0.0326\n",
      "Epoch 52/100\n",
      "4815/4815 [==============================] - 1s 257us/step - loss: 0.0393 - val_loss: 0.0347\n",
      "Epoch 53/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0393 - val_loss: 0.0313\n",
      "Epoch 54/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0393 - val_loss: 0.0345\n",
      "Epoch 55/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0407 - val_loss: 0.0330\n",
      "Epoch 56/100\n",
      "4815/4815 [==============================] - 1s 263us/step - loss: 0.0390 - val_loss: 0.0322\n",
      "Epoch 57/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0392 - val_loss: 0.0305\n",
      "Epoch 58/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0386 - val_loss: 0.0324\n",
      "Epoch 59/100\n",
      "4815/4815 [==============================] - 1s 270us/step - loss: 0.0395 - val_loss: 0.0317\n",
      "Epoch 60/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0395 - val_loss: 0.0319\n",
      "Epoch 61/100\n",
      "4815/4815 [==============================] - 1s 257us/step - loss: 0.0397 - val_loss: 0.0344\n",
      "Epoch 62/100\n",
      "4815/4815 [==============================] - 1s 265us/step - loss: 0.0394 - val_loss: 0.0325\n",
      "Epoch 63/100\n",
      "4815/4815 [==============================] - 1s 256us/step - loss: 0.0407 - val_loss: 0.0310\n",
      "Epoch 64/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0383 - val_loss: 0.0332\n",
      "Epoch 65/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0387 - val_loss: 0.0314\n",
      "Epoch 66/100\n",
      "4815/4815 [==============================] - 1s 261us/step - loss: 0.0386 - val_loss: 0.0322\n",
      "Epoch 67/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0381 - val_loss: 0.0310\n",
      "Epoch 00067: early stopping\n",
      "Test RMSE :  0.17609458235641634\n",
      "Running CV Iteration Num : 3\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/100\n",
      "4815/4815 [==============================] - 2s 488us/step - loss: 1.2260 - val_loss: 0.1995\n",
      "Epoch 2/100\n",
      "4815/4815 [==============================] - 1s 306us/step - loss: 0.3746 - val_loss: 0.1458\n",
      "Epoch 3/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.2373 - val_loss: 0.1345\n",
      "Epoch 4/100\n",
      "4815/4815 [==============================] - 1s 270us/step - loss: 0.1902 - val_loss: 0.1058\n",
      "Epoch 5/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.1558 - val_loss: 0.1042\n",
      "Epoch 6/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.1385 - val_loss: 0.0910\n",
      "Epoch 7/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.1291 - val_loss: 0.0846\n",
      "Epoch 8/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.1135 - val_loss: 0.0737\n",
      "Epoch 9/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.1099 - val_loss: 0.0798\n",
      "Epoch 10/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.1024 - val_loss: 0.0707\n",
      "Epoch 11/100\n",
      "4815/4815 [==============================] - 1s 270us/step - loss: 0.0965 - val_loss: 0.0684\n",
      "Epoch 12/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0911 - val_loss: 0.0586\n",
      "Epoch 13/100\n",
      "4815/4815 [==============================] - 2s 314us/step - loss: 0.0868 - val_loss: 0.0683\n",
      "Epoch 14/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0819 - val_loss: 0.0556\n",
      "Epoch 15/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0778 - val_loss: 0.0672\n",
      "Epoch 16/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0749 - val_loss: 0.0554\n",
      "Epoch 17/100\n",
      "4815/4815 [==============================] - 1s 267us/step - loss: 0.0696 - val_loss: 0.0571\n",
      "Epoch 18/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0671 - val_loss: 0.0499\n",
      "Epoch 19/100\n",
      "4815/4815 [==============================] - 1s 268us/step - loss: 0.0638 - val_loss: 0.0528\n",
      "Epoch 20/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0622 - val_loss: 0.0459\n",
      "Epoch 21/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0585 - val_loss: 0.0647\n",
      "Epoch 22/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0555 - val_loss: 0.0492\n",
      "Epoch 23/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0544 - val_loss: 0.0552\n",
      "Epoch 24/100\n",
      "4815/4815 [==============================] - 1s 269us/step - loss: 0.0518 - val_loss: 0.0465\n",
      "Epoch 25/100\n",
      "4815/4815 [==============================] - 1s 266us/step - loss: 0.0505 - val_loss: 0.0446\n",
      "Epoch 26/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0506 - val_loss: 0.0399\n",
      "Epoch 27/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0508 - val_loss: 0.0425\n",
      "Epoch 28/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0479 - val_loss: 0.0400\n",
      "Epoch 29/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.0456 - val_loss: 0.0436\n",
      "Epoch 30/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0460 - val_loss: 0.0412\n",
      "Epoch 31/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0456 - val_loss: 0.0412\n",
      "Epoch 32/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0453 - val_loss: 0.0402\n",
      "Epoch 33/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.0464 - val_loss: 0.0411\n",
      "Epoch 34/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0443 - val_loss: 0.0390\n",
      "Epoch 35/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0443 - val_loss: 0.0395\n",
      "Epoch 36/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0438 - val_loss: 0.0402\n",
      "Epoch 37/100\n",
      "4815/4815 [==============================] - 1s 260us/step - loss: 0.0416 - val_loss: 0.0399\n",
      "Epoch 38/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0432 - val_loss: 0.0391\n",
      "Epoch 39/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.0442 - val_loss: 0.0377\n",
      "Epoch 40/100\n",
      "4815/4815 [==============================] - 1s 260us/step - loss: 0.0424 - val_loss: 0.0420\n",
      "Epoch 41/100\n",
      "4815/4815 [==============================] - 1s 253us/step - loss: 0.0426 - val_loss: 0.0395\n",
      "Epoch 42/100\n",
      "4815/4815 [==============================] - 1s 262us/step - loss: 0.0416 - val_loss: 0.0406\n",
      "Epoch 43/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.0418 - val_loss: 0.0418\n",
      "Epoch 44/100\n",
      "4815/4815 [==============================] - 1s 263us/step - loss: 0.0420 - val_loss: 0.0383\n",
      "Epoch 45/100\n",
      "4815/4815 [==============================] - 1s 265us/step - loss: 0.0407 - val_loss: 0.0370\n",
      "Epoch 46/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0416 - val_loss: 0.0375\n",
      "Epoch 47/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0412 - val_loss: 0.0370\n",
      "Epoch 48/100\n",
      "4815/4815 [==============================] - 1s 303us/step - loss: 0.0412 - val_loss: 0.0381\n",
      "Epoch 49/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0407 - val_loss: 0.0382\n",
      "Epoch 50/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.0402 - val_loss: 0.0366\n",
      "Epoch 51/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0403 - val_loss: 0.0390\n",
      "Epoch 52/100\n",
      "4815/4815 [==============================] - 1s 265us/step - loss: 0.0397 - val_loss: 0.0360\n",
      "Epoch 53/100\n",
      "4815/4815 [==============================] - 1s 266us/step - loss: 0.0395 - val_loss: 0.0381\n",
      "Epoch 54/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.0391 - val_loss: 0.0366\n",
      "Epoch 55/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.0390 - val_loss: 0.0448\n",
      "Epoch 56/100\n",
      "4815/4815 [==============================] - 1s 267us/step - loss: 0.0391 - val_loss: 0.0379\n",
      "Epoch 57/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0382 - val_loss: 0.0356\n",
      "Epoch 58/100\n",
      "4815/4815 [==============================] - 1s 264us/step - loss: 0.0398 - val_loss: 0.0358\n",
      "Epoch 59/100\n",
      "4815/4815 [==============================] - 1s 267us/step - loss: 0.0380 - val_loss: 0.0378\n",
      "Epoch 60/100\n",
      "4815/4815 [==============================] - 1s 263us/step - loss: 0.0384 - val_loss: 0.0376\n",
      "Epoch 61/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0387 - val_loss: 0.0366\n",
      "Epoch 62/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0389 - val_loss: 0.0363\n",
      "Epoch 63/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0369 - val_loss: 0.0355\n",
      "Epoch 64/100\n",
      "4815/4815 [==============================] - 1s 270us/step - loss: 0.0377 - val_loss: 0.0357\n",
      "Epoch 65/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0387 - val_loss: 0.0359\n",
      "Epoch 66/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0386 - val_loss: 0.0369\n",
      "Epoch 67/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0389 - val_loss: 0.0366\n",
      "Epoch 68/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0384 - val_loss: 0.0361\n",
      "Epoch 69/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0378 - val_loss: 0.0374\n",
      "Epoch 70/100\n",
      "4815/4815 [==============================] - 1s 277us/step - loss: 0.0375 - val_loss: 0.0374\n",
      "Epoch 71/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.0371 - val_loss: 0.0371\n",
      "Epoch 72/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0382 - val_loss: 0.0377\n",
      "Epoch 73/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0376 - val_loss: 0.0372\n",
      "Epoch 00073: early stopping\n",
      "Test RMSE :  0.19298481068121467\n",
      "Running CV Iteration Num : 4\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/100\n",
      "4815/4815 [==============================] - 2s 516us/step - loss: 1.2761 - val_loss: 0.2094\n",
      "Epoch 2/100\n",
      "4815/4815 [==============================] - 1s 271us/step - loss: 0.3789 - val_loss: 0.1624\n",
      "Epoch 3/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.2484 - val_loss: 0.1272\n",
      "Epoch 4/100\n",
      "4815/4815 [==============================] - 1s 293us/step - loss: 0.1935 - val_loss: 0.1149\n",
      "Epoch 5/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.1625 - val_loss: 0.0834\n",
      "Epoch 6/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.1411 - val_loss: 0.0878\n",
      "Epoch 7/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.1253 - val_loss: 0.0553\n",
      "Epoch 8/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.1165 - val_loss: 0.0622\n",
      "Epoch 9/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.1094 - val_loss: 0.0492\n",
      "Epoch 10/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.1030 - val_loss: 0.0486\n",
      "Epoch 11/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.1002 - val_loss: 0.0636\n",
      "Epoch 12/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0917 - val_loss: 0.0442\n",
      "Epoch 13/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0854 - val_loss: 0.0465\n",
      "Epoch 14/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0832 - val_loss: 0.0463\n",
      "Epoch 15/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0778 - val_loss: 0.0396\n",
      "Epoch 16/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0746 - val_loss: 0.0533\n",
      "Epoch 17/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0703 - val_loss: 0.0387\n",
      "Epoch 18/100\n",
      "4815/4815 [==============================] - 1s 275us/step - loss: 0.0685 - val_loss: 0.0361\n",
      "Epoch 19/100\n",
      "4815/4815 [==============================] - 2s 314us/step - loss: 0.0652 - val_loss: 0.0395\n",
      "Epoch 20/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0616 - val_loss: 0.0374\n",
      "Epoch 21/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0601 - val_loss: 0.0347\n",
      "Epoch 22/100\n",
      "4815/4815 [==============================] - 1s 274us/step - loss: 0.0557 - val_loss: 0.0387\n",
      "Epoch 23/100\n",
      "4815/4815 [==============================] - 1s 282us/step - loss: 0.0568 - val_loss: 0.0355\n",
      "Epoch 24/100\n",
      "4815/4815 [==============================] - 1s 276us/step - loss: 0.0534 - val_loss: 0.0327\n",
      "Epoch 25/100\n",
      "4815/4815 [==============================] - 1s 273us/step - loss: 0.0520 - val_loss: 0.0401\n",
      "Epoch 26/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0505 - val_loss: 0.0321\n",
      "Epoch 27/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0506 - val_loss: 0.0337\n",
      "Epoch 28/100\n",
      "4815/4815 [==============================] - 1s 283us/step - loss: 0.0504 - val_loss: 0.0397\n",
      "Epoch 29/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0474 - val_loss: 0.0329\n",
      "Epoch 30/100\n",
      "4815/4815 [==============================] - 1s 279us/step - loss: 0.0478 - val_loss: 0.0328\n",
      "Epoch 31/100\n",
      "4815/4815 [==============================] - 1s 281us/step - loss: 0.0475 - val_loss: 0.0340\n",
      "Epoch 32/100\n",
      "4815/4815 [==============================] - 1s 280us/step - loss: 0.0466 - val_loss: 0.0329\n",
      "Epoch 33/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0461 - val_loss: 0.0323\n",
      "Epoch 34/100\n",
      "4815/4815 [==============================] - 1s 284us/step - loss: 0.0458 - val_loss: 0.0336\n",
      "Epoch 35/100\n",
      "4815/4815 [==============================] - 1s 278us/step - loss: 0.0461 - val_loss: 0.0351\n",
      "Epoch 36/100\n",
      "4815/4815 [==============================] - 1s 272us/step - loss: 0.0446 - val_loss: 0.0337\n",
      "Epoch 00036: early stopping\n",
      "Test RMSE :  0.1836074653059765\n",
      "Running CV Iteration Num : 5\n",
      "Train on 4816 samples, validate on 1203 samples\n",
      "Epoch 1/100\n",
      "4816/4816 [==============================] - 3s 528us/step - loss: 1.2415 - val_loss: 0.1951\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/100\n",
      "4816/4816 [==============================] - 1s 270us/step - loss: 0.3779 - val_loss: 0.1650\n",
      "Epoch 3/100\n",
      "4816/4816 [==============================] - 1s 272us/step - loss: 0.2460 - val_loss: 0.1097\n",
      "Epoch 4/100\n",
      "4816/4816 [==============================] - 1s 284us/step - loss: 0.1842 - val_loss: 0.1077\n",
      "Epoch 5/100\n",
      "4816/4816 [==============================] - 1s 270us/step - loss: 0.1592 - val_loss: 0.0705\n",
      "Epoch 6/100\n",
      "4816/4816 [==============================] - 1s 260us/step - loss: 0.1416 - val_loss: 0.0732\n",
      "Epoch 7/100\n",
      "4816/4816 [==============================] - 1s 277us/step - loss: 0.1266 - val_loss: 0.0687\n",
      "Epoch 8/100\n",
      "4816/4816 [==============================] - 1s 304us/step - loss: 0.1176 - val_loss: 0.0532\n",
      "Epoch 9/100\n",
      "4816/4816 [==============================] - 1s 264us/step - loss: 0.1097 - val_loss: 0.0437\n",
      "Epoch 10/100\n",
      "4816/4816 [==============================] - 1s 285us/step - loss: 0.1038 - val_loss: 0.0416\n",
      "Epoch 11/100\n",
      "4816/4816 [==============================] - 1s 265us/step - loss: 0.0973 - val_loss: 0.0440\n",
      "Epoch 12/100\n",
      "4816/4816 [==============================] - 1s 269us/step - loss: 0.0900 - val_loss: 0.0442\n",
      "Epoch 13/100\n",
      "4816/4816 [==============================] - 1s 275us/step - loss: 0.0862 - val_loss: 0.0482\n",
      "Epoch 14/100\n",
      "4816/4816 [==============================] - 1s 267us/step - loss: 0.0826 - val_loss: 0.0428\n",
      "Epoch 15/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0768 - val_loss: 0.0425\n",
      "Epoch 16/100\n",
      "4816/4816 [==============================] - 1s 274us/step - loss: 0.0735 - val_loss: 0.0364\n",
      "Epoch 17/100\n",
      "4816/4816 [==============================] - 1s 273us/step - loss: 0.0681 - val_loss: 0.0362\n",
      "Epoch 18/100\n",
      "4816/4816 [==============================] - 1s 283us/step - loss: 0.0698 - val_loss: 0.0366\n",
      "Epoch 19/100\n",
      "4816/4816 [==============================] - 1s 282us/step - loss: 0.0631 - val_loss: 0.0352\n",
      "Epoch 20/100\n",
      "4816/4816 [==============================] - 1s 274us/step - loss: 0.0604 - val_loss: 0.0402\n",
      "Epoch 21/100\n",
      "4816/4816 [==============================] - 1s 277us/step - loss: 0.0601 - val_loss: 0.0340\n",
      "Epoch 22/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0557 - val_loss: 0.0367\n",
      "Epoch 23/100\n",
      "4816/4816 [==============================] - 1s 265us/step - loss: 0.0554 - val_loss: 0.0361\n",
      "Epoch 24/100\n",
      "4816/4816 [==============================] - 1s 275us/step - loss: 0.0547 - val_loss: 0.0298\n",
      "Epoch 25/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0542 - val_loss: 0.0277\n",
      "Epoch 26/100\n",
      "4816/4816 [==============================] - 1s 304us/step - loss: 0.0511 - val_loss: 0.0347\n",
      "Epoch 27/100\n",
      "4816/4816 [==============================] - 1s 277us/step - loss: 0.0478 - val_loss: 0.0287\n",
      "Epoch 28/100\n",
      "4816/4816 [==============================] - 1s 272us/step - loss: 0.0507 - val_loss: 0.0330\n",
      "Epoch 29/100\n",
      "4816/4816 [==============================] - 1s 286us/step - loss: 0.0484 - val_loss: 0.0288\n",
      "Epoch 30/100\n",
      "4816/4816 [==============================] - 1s 291us/step - loss: 0.0486 - val_loss: 0.0298\n",
      "Epoch 31/100\n",
      "4816/4816 [==============================] - 1s 277us/step - loss: 0.0466 - val_loss: 0.0327\n",
      "Epoch 32/100\n",
      "4816/4816 [==============================] - 1s 271us/step - loss: 0.0469 - val_loss: 0.0275\n",
      "Epoch 33/100\n",
      "4816/4816 [==============================] - 1s 277us/step - loss: 0.0446 - val_loss: 0.0267\n",
      "Epoch 34/100\n",
      "4816/4816 [==============================] - 1s 274us/step - loss: 0.0445 - val_loss: 0.0283\n",
      "Epoch 35/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0452 - val_loss: 0.0279\n",
      "Epoch 36/100\n",
      "4816/4816 [==============================] - 1s 271us/step - loss: 0.0443 - val_loss: 0.0279\n",
      "Epoch 37/100\n",
      "4816/4816 [==============================] - 1s 269us/step - loss: 0.0455 - val_loss: 0.0283\n",
      "Epoch 38/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0458 - val_loss: 0.0274\n",
      "Epoch 39/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0443 - val_loss: 0.0254\n",
      "Epoch 40/100\n",
      "4816/4816 [==============================] - 1s 275us/step - loss: 0.0430 - val_loss: 0.0270\n",
      "Epoch 41/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0438 - val_loss: 0.0252\n",
      "Epoch 42/100\n",
      "4816/4816 [==============================] - 1s 281us/step - loss: 0.0455 - val_loss: 0.0275\n",
      "Epoch 43/100\n",
      "4816/4816 [==============================] - 1s 273us/step - loss: 0.0426 - val_loss: 0.0259\n",
      "Epoch 44/100\n",
      "4816/4816 [==============================] - 1s 274us/step - loss: 0.0430 - val_loss: 0.0264\n",
      "Epoch 45/100\n",
      "4816/4816 [==============================] - 1s 280us/step - loss: 0.0422 - val_loss: 0.0250\n",
      "Epoch 46/100\n",
      "4816/4816 [==============================] - 1s 281us/step - loss: 0.0419 - val_loss: 0.0272\n",
      "Epoch 47/100\n",
      "4816/4816 [==============================] - 1s 274us/step - loss: 0.0420 - val_loss: 0.0259\n",
      "Epoch 48/100\n",
      "4816/4816 [==============================] - 1s 282us/step - loss: 0.0419 - val_loss: 0.0245\n",
      "Epoch 49/100\n",
      "4816/4816 [==============================] - 1s 255us/step - loss: 0.0422 - val_loss: 0.0257\n",
      "Epoch 50/100\n",
      "4816/4816 [==============================] - 1s 272us/step - loss: 0.0419 - val_loss: 0.0283\n",
      "Epoch 51/100\n",
      "4816/4816 [==============================] - 1s 269us/step - loss: 0.0404 - val_loss: 0.0257\n",
      "Epoch 52/100\n",
      "4816/4816 [==============================] - 1s 275us/step - loss: 0.0408 - val_loss: 0.0255\n",
      "Epoch 53/100\n",
      "4816/4816 [==============================] - 2s 421us/step - loss: 0.0412 - val_loss: 0.0269\n",
      "Epoch 54/100\n",
      "4816/4816 [==============================] - 1s 270us/step - loss: 0.0403 - val_loss: 0.0262\n",
      "Epoch 55/100\n",
      "4816/4816 [==============================] - 1s 266us/step - loss: 0.0409 - val_loss: 0.0254\n",
      "Epoch 56/100\n",
      "4816/4816 [==============================] - 1s 273us/step - loss: 0.0405 - val_loss: 0.0247\n",
      "Epoch 57/100\n",
      "4816/4816 [==============================] - 1s 268us/step - loss: 0.0410 - val_loss: 0.0259\n",
      "Epoch 58/100\n",
      "4816/4816 [==============================] - 1s 276us/step - loss: 0.0403 - val_loss: 0.0266\n",
      "Epoch 00058: early stopping\n",
      "Test RMSE :  0.16306886425910733\n",
      "CV RMSE :  0.17607317726030516\n"
     ]
    }
   ],
   "source": [
    "IterationNum = 1\n",
    "for train_index, test_index in kf.split(normed_x_train02):\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    MOD_DATA_2_TRAIN, MOD_DATA_2_TEST = train01.iloc[train_index], train01.iloc[test_index]\n",
    "    X_TRAIN, X_TEST = normed_x_train02.iloc[train_index], normed_x_train02.iloc[test_index]\n",
    "    Y_TRAIN, Y_TEST = y_train02[train_index], y_train02[test_index]\n",
    "    \n",
    "    model = nn_model()\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "    fin_model = model.fit(X_TRAIN, Y_TRAIN, validation_data = (X_TEST, Y_TEST), epochs = 100, batch_size = 8, verbose = 1, callbacks=[es])\n",
    "    \n",
    "    MOD_DATA_2_TEST['Predicted_Model_Value'] = model.predict(X_TEST)\n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = MOD_DATA_2_TEST.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = model.predict(normed_x_sub2)\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,MOD_DATA_2_TEST])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = test_scored + model.predict(normed_x_sub2)\n",
    "                    \n",
    "    IterationNum = IterationNum + 1\n",
    "    \n",
    "    print(\"Test RMSE : \",sqrt(mean_squared_error(MOD_DATA_2_TEST[\"Price\"], MOD_DATA_2_TEST['Predicted_Model_Value'])))\n",
    "    \n",
    "print(\"CV RMSE : \",sqrt(mean_squared_error(CV_SCORED_DATA[\"Price\"], CV_SCORED_DATA['Predicted_Model_Value'])))\n",
    "#CV RMSE :  0.17607317726030516"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=1234, minmax=(array([0.8876152], dtype=float32), array([3.7292836], dtype=float32)), mean=array([1.9737476], dtype=float32), variance=array([0.53152084], dtype=float32), skewness=array([0.7217], dtype=float32), kurtosis=array([-0.2060821], dtype=float32))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "test_scored2 = test_scored / 5.0\n",
    "stats.describe(test_scored2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.477000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.579717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18.617081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.126414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.404558</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Price\n",
       "0   2.477000\n",
       "1   2.579717\n",
       "2  18.617081\n",
       "3   4.126414\n",
       "4   4.404558"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_scored3 = pd.DataFrame({'Price' : np.exp(test_scored2[:,0])-1})\n",
    "test_scored3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV_SCORED_DATA.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\CV_Scored\\\\20190716_Keras01_CVTRAIN_DS.csv\",\n",
    "                      index = False)\n",
    "test_scored3.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\Submission\\\\20190716_Keras01_TEST_DS.csv\",\n",
    "                    index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
